import pandas as pd
import json
import numpy as np
# import scipy as sp
import matplotlib.pyplot as plt
import os, re, sys, glob, math
import warnings; warnings.simplefilter('ignore')
# import statsmodels.api as sm # import statsmodels
import lzma
import base64
from statistics import mean, median
from matplotlib.ticker import PercentFormatter
# from datetime import datetime

#Import of data

subfolder = 'Base-Salient-Strategy'
outputpath = f'../output/{subfolder}/'
outputpathErrors = f'../output/{subfolder}/errors/'

# Cleaning data
def merge_all_files(keyword,subfolder='main'):
    all_files = glob.glob(os.path.join(r'../data/'+subfolder+'/', keyword + '*.csv'))
    return pd.concat((pd.read_csv(f) for f in all_files), ignore_index=True, sort=True)

data = merge_all_files('inference',subfolder=subfolder)
sessions = ['yt064y3k', 'pe3pb92c']
# taskDifficultyValues = [1,2]
# Only keep those in-session who finished
data = data[data['session.code'].isin(sessions) & data['participant._current_page_name'].isin(['Redirect'])]
print(len(data))